import re
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.contrib.spiders import Rule
from scrapy.contrib.spiders.crawl import CrawlSpider
from scrapy.selector.libxml2sel import HtmlXPathSelector

from __init__ import *
from kateglo_crawler.items import DictionaryItem


#
# DICTIONARY LIST CRAWL
#
class DictionaryListCrawler(CrawlSpider):
    # Identifier
    name = 'dictionary_list'

    # Debug
    debug = False

    # Rules
    allowed_domains = ['kateglo.bahtera.org']
    start_urls = [
        'http://kateglo.bahtera.org/?mod=dictionary&srch=all',
    ]
    rules = (
        Rule(SgmlLinkExtractor(allow=(re.compile('&srch=all')), deny=('action=view'), unique=True), follow=True, callback='parse_item'),
    )

    def parse_item(self, response):
        hxs = HtmlXPathSelector(response)

        # Assign given elements
        dictionary_items = []
        phrases = hxs.select('//p/b/a/text()').extract()
        for phrase in phrases:
            dictionary_item = DictionaryItem()
            dictionary_item['phrase'] = phrase
            dictionary_item['urllist'] = response.url
            dictionary_item['isdetailed'] = 0
            dictionary_items.append(dictionary_item)
        return dictionary_items